structure Tokens =
struct 

datatype token =
	 ARROW of int
       | DOT of int 
       | COMMA of int
       | SEMICOLON of int
       | LPAREN of int
       | RPAREN of int
       | LBRACE of int
       | RBRACE of int
       | LBRACK of int
       | RBRACK of int
       | PLUS of int
       | MINUS of int
       | TIMES of int
       | DIVIDE of int
       | MODUS of int
       | GE of int
       | GT of int
       | LT of int
       | LE of int
       | EQ of int
       | NEQ of int
       | AND of int
       | OR of int
       | NOT of int
       | ASSIGN of int
       | ENUM of int
       | INT of int
       | CHAR of int
       | VOID of int
       | IF of int
       | FOR of int
       | ELSE of int
       | WHILE of int
       | STRUCT of int
       | RETURN of int
       | STRINGCONST of string * int
       | ID of string * int 
       | INTCONST of string * int
       | EOF

fun helper str int =
    str ^ " AT POS:" ^ Int.toString int

fun toString token =
    case token of 
	ARROW pos => helper "->" pos
      | DOT pos => helper "." pos
      | COMMA pos => helper "," pos
      | SEMICOLON pos => helper ";" pos
      | LPAREN pos => helper "(" pos
      | RPAREN pos => helper ")" pos
      | LBRACE pos => helper "{" pos
      | RBRACE pos => helper "}" pos
      | LBRACK pos => helper "[" pos
      | RBRACK pos => helper "]" pos
      | PLUS pos => helper "+" pos
      | MINUS pos => helper "-" pos
      | TIMES pos => helper "*" pos
      | DIVIDE pos => helper "/" pos
      | MODUS pos => helper "%" pos
      | GE pos => helper ">=" pos
      | GT pos => helper ">" pos
      | LT pos => helper "<" pos
      | LE pos => helper "<=" pos
      | EQ pos => helper "==" pos
      | NEQ pos => helper "!=" pos
      | AND pos => helper "&&" pos
      | OR pos => helper "||" pos
      | NOT pos => helper "!" pos
      | ASSIGN pos => helper "=" pos
      | ENUM pos => helper "enum" pos
      | INT pos => helper "int" pos
      | CHAR pos => helper "char" pos
      | VOID pos => helper "void" pos
      | IF pos => helper "if" pos
      | FOR pos => helper "for" pos
      | ELSE pos => helper "else" pos
      | WHILE pos => helper "while" pos
      | STRUCT pos => helper "struct" pos
      | RETURN pos => helper "return" pos
      | ID(text,pos) => helper ("ID("^text^")") pos 
      | INTCONST(int,pos) => helper ("INT("^int^")") pos 
      | STRINGCONST(str,pos) => helper ("STRING("^str^")") pos 
      | EOF => ""

end

